# 作者 :南雨
# 时间 : 2022/3/23 20:50
# 创建词典
import re


# 加载数据
def loadData(base_path):
    data_set = []
    with open(base_path, "rb") as f:
        line = f.readline()
        while line:
            line = line.decode("utf-8", "ignore")
            data_set.append(line)
            line = f.readline()
    return data_set


# 预处理数据
def preDO(data_set):
    sentence_list = []
    for sen in data_set:
        sentence = sen.split("。" or "？" or "！")
        sentence = sentence
        sentence_list.extend(sentence)

    return sentence_list


# 生成词典
def getDict(sentence_list):
    word_dict = []
    test_sentence_list = []
    word_test_labels = []

    rule = ['！', '”', '#', '$', '%', '&', '（', '）', '\*', '\+', '，', '——', '。', '/', '：', '；', '《', '=', '》',
            '？', '@', '【', '\\', '】', '--', '、', '`', '\{', '\|', '\}', '~', '\t', '\n', '\x97', '\x96', '”',
            '“', ]

    for i in range(len(sentence_list)):
        sentence = sentence_list[i]
        sentence = sentence.strip()
        sentence1 = re.sub("|".join(rule), "", sentence)

        if not sentence1:
            continue
        words = sentence1.split("  ")

        for word in words:
            if word == '':
                words.remove(word)
        if len(sentence_list) - i > 60:
            word_dict.extend(words)
        else:
            test_sentence = ''.join(words)
            # 测试时用于分词的句子
            test_sentence_list.append(test_sentence)
            # 测试时用的标准的分词
            word_test_labels.append(words)
    return word_dict, test_sentence_list, word_test_labels


def save(base_path):
    data_set = loadData(base_path)
    sentence_list = preDO(data_set)
    result_tuple = getDict(sentence_list)
    return result_tuple
